package net.introlab.scraper.scanner;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import net.introlab.scraper.domain.filter.BaseFilter;
import net.introlab.scraper.exception.ScraperException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.*;
import java.util.logging.Level;

public abstract class BaseJsoupScanner<T, V extends BaseFilter> implements Scanner<T, V> {

    static {
        LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
        java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(Level.OFF);
    }

    private List<T> dataList = new ArrayList<>();
    protected int pageCounter = 0; // Page from which parsing starting
    protected int lastPageToScan = -1; // End of parsing pages range
    protected int numberOfItemsToScan = -1;
    protected V filter;

    protected final WebClient webClient;
    protected HtmlPage lastRequestedPage;
//    private int windowHeight = 1080;
//
//    public BaseJsoupScanner(int windowHeight) {
//        this();
//        this.windowHeight = windowHeight;
//    }

    public BaseJsoupScanner() {
        this.webClient = getWebClient();
    }

    @Override
    public List<T> scan(V filter) {
        try {
            this.filter = filter;
            numberOfItemsToScan = filter.getNumberOfItemsToScan();
            pageCounter = filter.getFirstPageToScan();
            lastPageToScan = filter.getLastPageToScan();
            scrapAllData(getPage(pageCounter, getURLWithFilter()));
            return dataList;
        } catch (IOException e) {
            throw new ScraperException(e);
        }
    }

    private String getURLWithFilter() {
        return applyFilter(getSiteURL(), filter);
    }

    protected abstract String applyFilter(String siteURL, V filter);

    private void scrapAllData(Document htmlPage) throws IOException {
        if (!isProcessCurrentPage(htmlPage)) {
            return;
        }
        dataList.addAll(scrapPageData(htmlPage));
        pageCounter++;
        if (processNextPage(htmlPage) && dataList.size() < numberOfItemsToScan && (pageCounter <= lastPageToScan || lastPageToScan == -1)) {
            scrapAllData(getNextPage(pageCounter));
        }
    }

    protected List<T> scrapPageData(Document doc) throws IOException {
        Elements elements = getElements(doc);
        int skipItems = getNumberOfItemsToSkip();
        List<T> pageDataList = new ArrayList<>();
        ExecutorService executorService = Executors.newFixedThreadPool(20);

        Set<Callable<T>> callables = new HashSet<>();
        List<Future<T>> dataFutureList;
        for (int i = skipItems; i < elements.size(); i++) {
            if (pageDataList.size() >= numberOfItemsToScan && numberOfItemsToScan != -1) {
                break;
            }
            Element element = elements.get(i);
            callables.add(getDataScraper(element));
        }
        try {
            dataFutureList = executorService.invokeAll(callables);
            for (Future<T> dataFuture : dataFutureList) {
                pageDataList.add(dataFuture.get());
            }
        } catch (InterruptedException | ExecutionException e) {
            e.printStackTrace();
            throw new ScraperException(e);
        }

        return pageDataList;
    }

    protected int getNumberOfItemsToSkip() {
        return 0;
    }

    protected Document getNextPage(int pageCounter) throws IOException {
        return getPage(pageCounter, getURLWithFilter());
    }

    protected abstract Document getPage(int pageNumber, String baseURI);

    protected abstract boolean processNextPage(Document htmlPage);

    protected abstract boolean isProcessCurrentPage(Document htmlPage);

    protected abstract Callable<T> getDataScraper(Element element);

    protected abstract Elements getElements(Document doc);

    protected String getParametersDelimiter(String baseURI) {
        return StringUtils.contains(baseURI, "?") ? "&" : "?";
    }

    public List<T> getDataList() {
        return dataList;
    }

    protected Document connect(String site) {
        try {
            return Jsoup.connect(site).timeout(20000).get();
        } catch (IOException e) {
            throw new ScraperException(e);
        }
    }

//    protected String getElementValue(Element element) {
//        return element.child(0).html();
//    }

    protected HtmlPage getHtmlPage(String url) {
        return getHtmlPage(url, webClient);
    }

    protected HtmlPage getNewHtmlPage(String link) throws IOException {
        return getHtmlPage(link, getWebClient());
    }

    private HtmlPage getHtmlPage(String url, WebClient webClient) {
        try {
            HtmlPage page = webClient.getPage(url);
            lastRequestedPage = page;
            waitForScript(10000);
            return page;
        } catch (IOException e) {
            throw new ScraperException(e);
        }
    }

    private WebClient getWebClient() {
        WebClient client = new WebClient(BrowserVersion.CHROME);
        client.getOptions().setJavaScriptEnabled(true);
        client.getOptions().setThrowExceptionOnScriptError(false);
        client.getOptions().setThrowExceptionOnFailingStatusCode(false);
        client.waitForBackgroundJavaScript(1500000); //Wait for document.ready Auto search to fire and fetch page results via AJAX
        client.setAjaxController(new NicelyResynchronizingAjaxController());
        client.waitForBackgroundJavaScriptStartingBefore(1500000);
//        client.getCurrentWindow().setInnerHeight(windowHeight);
        return client;
    }

    protected void waitForScript(int time) {
        webClient.waitForBackgroundJavaScript(time);
    }

    protected Document parseHtml(String html) {
        return Jsoup.parse(html);
    }
}
